'''
Created on Mar 10, 2013

@author: peterd

Retry handler integrated with ippool proxy manager
'''

import os
import re

from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware
from scrapy import log

from ippool import host_mgr 

global dbg_activated
dbg_activated = False

class ProxyRetryMiddleware(RetryMiddleware):
    def __init__(self, settings):
        RetryMiddleware.__init__(self, settings)
        
    def _retry(self, request, reason, spider):
        log.msg(format='from ProxyRetryMiddleware::_retry(), proxy=%(proxy)s', level=log.DEBUG, proxy=request.meta['proxy'])
            
        #=======================================================================
        # global dbg_activated
        # if not dbg_activated:                        
        #    import pdb;pdb.set_trace()
        #    dbg_activated = True
        # 
        #=======================================================================
        retries = request.meta.get('retry_times', 0) + 1

        if retries <= self.max_retry_times:
            proxy = request.meta['proxy']
            
            if proxy:
                m = re.search('http://(.*?):(\d+)', proxy)
                if m:
                    is_proxy_alive = host_mgr.is_alive(m.group(1), m.group(2))
                    
                    if is_proxy_alive:                        
                        log.msg(format="Retrying %(request)s (failed %(retries)d times): %(reason)s",
                                level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason)
                        retryreq = request.copy()
                        retryreq.meta['retry_times'] = retries
                        retryreq.dont_filter = True
                        retryreq.priority = request.priority + self.priority_adjust
                        return retryreq
                    else:
                        # get new proxy
                        proxy = host_mgr.get_host()
                        if proxy is not None:                                    
                            log.msg(format="Retrying %(request)s (failed %(retries)d times): %(reason)s Setting http proxy to %(proxy)s",
                                level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason, proxy=proxy)
                            retryreq = request.copy()
                            retryreq.meta['proxy'] = 'http://%s:%s' % (proxy['ipaddress'], proxy['port'])
                            retryreq.meta['retry_times'] = retries
                            retryreq.dont_filter = True
                            retryreq.priority = request.priority + self.priority_adjust
                            os.environ['live_http_proxy'] = retryreq.meta['proxy'] 
                            return retryreq
                        else:
                            log.msg(format="Gave up retrying %(request)s (failed %(retries)d times): %(reason)s Failed to obtain a proxy from ippool.",  
                                level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason)                            
            else:
                log.msg(format="Gave up retrying %(request)s (failed %(retries)d times): %(reason)s Proxy not set.",  
                    level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason)                                                  
        else:
            #import pdb;pdb.set_trace()
            # mark current proxy as dead
            proxy = request.meta['proxy']
            if proxy:
                m = re.search('http://(.*?):(\d+)', proxy)
                if m:
                    host_mgr.set_is_alive(m.group(1), m.group(2), False)
            # get new proxy
            proxy = host_mgr.get_host()
            if proxy is not None:                                    
                log.msg(format="Gave up retrying %(request)s (failed %(retries)d times): %(reason)s Setting http proxy to %(proxy)s and retrying again.",
                    level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason, proxy=proxy)
                retryreq = request.copy()
                retryreq.meta['proxy'] = 'http://%s:%s' % (proxy['ipaddress'], proxy['port'])
                retryreq.meta['retry_times'] = 0
                retryreq.dont_filter = True
                retryreq.priority = request.priority + self.priority_adjust
                os.environ['live_http_proxy'] = retryreq.meta['proxy']
                return retryreq
            else:
                log.msg(format="Gave up retrying %(request)s (failed %(retries)d times): %(reason)s Failed to obtain a proxy from ippool.",  
                    level=log.DEBUG, spider=spider, request=request, retries=retries, reason=reason)                                  